{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 32.487702,
     "end_time": "2023-10-05T14:37:28.610784",
     "exception": false,
     "start_time": "2023-10-05T14:36:56.123082",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from gc import collect;\n",
    "from warnings import filterwarnings;\n",
    "filterwarnings('ignore');\n",
    "from IPython.display import clear_output;\n",
    "\n",
    "!pip install -q --upgrade scipy;\n",
    "!pip install -q category_encoders;\n",
    "\n",
    "clear_output();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 1.504753,
     "end_time": "2023-10-05T14:37:30.126100",
     "exception": false,
     "start_time": "2023-10-05T14:37:28.621347",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from copy import deepcopy;\n",
    "import pandas as pd;\n",
    "import numpy as np;\n",
    "import re;\n",
    "from scipy.stats import mode, kstest, normaltest, shapiro, anderson, jarque_bera;\n",
    "from collections import Counter;\n",
    "from itertools import product;\n",
    "from colorama import Fore, Style, init;\n",
    "from warnings import filterwarnings;\n",
    "filterwarnings('ignore');\n",
    "\n",
    "from tqdm.notebook import tqdm;\n",
    "import seaborn as sns;\n",
    "import matplotlib.pyplot as plt;\n",
    "from matplotlib.colors import ListedColormap as LCM;\n",
    "%matplotlib inline\n",
    "\n",
    "from pprint import pprint;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 4.906224,
     "end_time": "2023-10-05T14:37:35.042785",
     "exception": false,
     "start_time": "2023-10-05T14:37:30.136561",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from category_encoders import OrdinalEncoder, OneHotEncoder;\n",
    "from sklearn.preprocessing import (RobustScaler, \n",
    "                                   MinMaxScaler, \n",
    "                                   StandardScaler, \n",
    "                                   FunctionTransformer as FT\n",
    "                                  );\n",
    "from sklearn.impute import SimpleImputer as SI;\n",
    "from sklearn.model_selection import (RepeatedStratifiedKFold as RSKF, \n",
    "                                     StratifiedKFold as SKF,\n",
    "                                     KFold, \n",
    "                                     RepeatedKFold as RKF, \n",
    "                                     cross_val_score);\n",
    "from sklearn.inspection import permutation_importance;\n",
    "from sklearn.feature_selection import mutual_info_classif, RFE;\n",
    "from sklearn.pipeline import Pipeline, make_pipeline;\n",
    "from sklearn.base import BaseEstimator, TransformerMixin;\n",
    "from sklearn.compose import ColumnTransformer;\n",
    "\n",
    "from sklearn.metrics import roc_auc_score,  make_scorer;\n",
    "from xgboost import DMatrix, XGBClassifier;\n",
    "from lightgbm import LGBMClassifier, log_evaluation, early_stopping;\n",
    "from catboost import CatBoostClassifier, Pool;\n",
    "from sklearn.ensemble import (RandomForestClassifier as RFC, \n",
    "                              ExtraTreesClassifier as ETC,\n",
    "                              AdaBoostClassifier as ABC,\n",
    "                              BaggingClassifier as BC,\n",
    "                              HistGradientBoostingClassifier as HGBC,\n",
    "                              GradientBoostingClassifier as GBC,\n",
    "                              BaggingClassifier as BC,\n",
    "                             );\n",
    "from sklearn.neighbors import KNeighborsClassifier as KNNC;\n",
    "from sklearn.linear_model import LogisticRegression as LC;\n",
    "from sklearn.calibration import CalibrationDisplay as Clb;\n",
    "\n",
    "import optuna;\n",
    "from optuna import Trial, trial, create_study;\n",
    "from optuna.samplers import TPESampler, CmaEsSampler;\n",
    "optuna.logging.set_verbosity = optuna.logging.ERROR;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.222227,
     "end_time": "2023-10-05T14:37:35.276495",
     "exception": false,
     "start_time": "2023-10-05T14:37:35.054268",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "sns.set({\"axes.facecolor\"       : \"#ffffff\",\n",
    "         \"figure.facecolor\"     : \"#ffffff\",\n",
    "         \"axes.edgecolor\"       : \"#000000\",\n",
    "         \"grid.color\"           : \"#ffffff\",\n",
    "         \"font.family\"          : ['Cambria'],\n",
    "         \"axes.labelcolor\"      : \"#000000\",\n",
    "         \"xtick.color\"          : \"#000000\",\n",
    "         \"ytick.color\"          : \"#000000\",\n",
    "         \"grid.linewidth\"       : 0.75,  \n",
    "         \"grid.linestyle\"       : \"--\",\n",
    "         \"axes.titlecolor\"      : '#0099e6',\n",
    "         'axes.titlesize'       : 8.5,\n",
    "         'axes.labelweight'     : \"bold\",\n",
    "         'legend.fontsize'      : 7.0,\n",
    "         'legend.title_fontsize': 7.0,\n",
    "         'font.size'            : 7.5,\n",
    "         'xtick.labelsize'      : 7.5,\n",
    "         'ytick.labelsize'      : 7.5,        \n",
    "        });\n",
    " \n",
    "def PrintColor(text:str, color = Fore.BLUE, style = Style.BRIGHT):\n",
    "    \"Prints color outputs using colorama using a text F-string\";\n",
    "    print(style + color + text + Style.RESET_ALL); \n",
    "\n",
    "from sklearn import set_config; \n",
    "set_config(transform_output = \"pandas\");\n",
    "pd.set_option('display.max_columns', 50);\n",
    "pd.set_option('display.max_rows', 50);\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.223699,
     "end_time": "2023-10-05T14:37:35.573360",
     "exception": false,
     "start_time": "2023-10-05T14:37:35.349661",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 参数配置类以及用于调整和训练的CV策略\n",
    "class CFG:\n",
    "    # 数据准备\n",
    "    version_nb         = 3;\n",
    "    test_req           = \"N\";\n",
    "    # GPU型号\n",
    "    gpu_switch         = \"OFF\"; \n",
    "    # 随机数种子\n",
    "    state              = 42;\n",
    "    # 目标列名\n",
    "    target             = 'defects';\n",
    "    # 系列序号\n",
    "    episode            = 23;\n",
    "    # 比赛数据文件夹路径\n",
    "    path               = f\"/kaggle/input/playground-series-s3e{episode}/\";\n",
    "    # 原始数据文件路径\n",
    "    orig_path          = f\"/kaggle/input/software-defect-prediction/jm1.csv\";\n",
    "    \n",
    "    # 是否打印各列基本信息\n",
    "    dtl_preproc_req    = \"Y\";\n",
    "    adv_cv_req         = \"N\";\n",
    "    # 是否绘制各列分布情况图\n",
    "    ftre_plots_req     = 'Y';\n",
    "    # 是否绘制各列相关性图\n",
    "    ftre_imp_req       = \"Y\";\n",
    "    \n",
    "    # 数据转换和缩放   \n",
    "    # 是否加入原始数据\n",
    "    conjoin_orig_data  = \"Y\";\n",
    "    # 是否删除空值\n",
    "    drop_nulls         = \"Y\";\n",
    "    sec_ftre_req       = \"N\";\n",
    "    # 是否进行数据缩放\n",
    "    scale_req          = \"N\";\n",
    "    # 即使scale_req=N，也要在此处保留一个值，用于线性模型\n",
    "    scl_method         = \"Robust\"; \n",
    "    enc_method         = 'Label';\n",
    "    OH_cols            = [];\n",
    "    \n",
    "    # 模型训练\n",
    "    baseline_req       = \"N\";\n",
    "    pstprcs_oof        = \"N\";\n",
    "    # 是否将训练集规整到0-1之间（回归）\n",
    "    pstprcs_train      = \"N\";\n",
    "    # 是否将测试集规整到0-1之间（回归）\n",
    "    pstprcs_test       = \"Y\";\n",
    "    # 是否训练模型\n",
    "    ML                 = \"Y\";\n",
    "    # 是否将原始数据加入到CV环节\n",
    "    use_orig_allfolds  = \"N\";\n",
    "    # 交叉检验折数\n",
    "    n_splits           = 10 ;\n",
    "    # RSKF交叉检验时数据重复次数\n",
    "    n_repeats          = 1 ;\n",
    "    # 早停策略容忍步数\n",
    "    nbrnd_erly_stp     = 100 ;\n",
    "    # 交叉检验方法\n",
    "    mdlcv_mthd         = 'RSKF';\n",
    "    \n",
    "    # 模型组合    \n",
    "    # 是否使用模型组合\n",
    "    ensemble_req       = \"Y\";\n",
    "    # 模型组合交叉检验方法\n",
    "    enscv_mthd         = \"RSKF\";\n",
    "    # optuna搜索方向（最大化/最小化目标值）\n",
    "    metric_obj         = 'maximize';\n",
    "    # 搜索次数\n",
    "    ntrials            = 10 if test_req == \"Y\" else 1000;\n",
    "    \n",
    "    # 打印规范\n",
    "    # 表格打印规范\n",
    "    grid_specs = {'visible': True, 'which': 'both', 'linestyle': '--', \n",
    "                           'color': 'lightgrey', 'linewidth': 0.75};\n",
    "    # 标题打印规范\n",
    "    title_specs = {'fontsize': 9, 'fontweight': 'bold', 'color': 'tab:blue'};\n",
    "\n",
    "PrintColor(f\"--> Configuration done!\\n\");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.215259,
     "end_time": "2023-10-05T14:37:35.799286",
     "exception": false,
     "start_time": "2023-10-05T14:37:35.584027",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 定义要在整个代码中用于常见任务的函数\n",
    "\n",
    "# 用于连续列的缩放器\n",
    "all_scalers = {'Robust': RobustScaler(), \n",
    "               'Z': StandardScaler(), \n",
    "               'MinMax': MinMaxScaler()\n",
    "              };\n",
    "scaler      = all_scalers.get(CFG.scl_method);\n",
    "\n",
    "# 定义常用CV策略:-\n",
    "all_cv= {'KF'  : KFold(n_splits= CFG.n_splits, shuffle = True, random_state= CFG.state),\n",
    "         'RKF' : RKF(n_splits= CFG.n_splits, n_repeats = CFG.n_repeats, random_state= CFG.state),\n",
    "         'RSKF': RSKF(n_splits= CFG.n_splits, n_repeats = CFG.n_repeats, random_state= CFG.state),\n",
    "         'SKF' : SKF(n_splits= CFG.n_splits, shuffle = True, random_state= CFG.state)\n",
    "        };\n",
    "\n",
    "# 定义对比指标\n",
    "def ScoreMetric(ytrue, ypred)-> float:\n",
    "    return roc_auc_score(ytrue, ypred);\n",
    "\n",
    "# 将结果更正到0~1\n",
    "def PostProcessPred(preds, post_process = \"N\"):\n",
    "    if post_process == \"Y\":\n",
    "        return np.clip(preds, a_min = 0.0, a_max= 1.0);\n",
    "    else:\n",
    "        return preds;\n",
    "\n",
    "# 设计一个自定义记分器以用于cross_val_epredict和cross_val_score\n",
    "myscorer = make_scorer(ScoreMetric, greater_is_better = True, needs_proba= True,);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.236543,
     "end_time": "2023-10-05T14:37:36.201295",
     "exception": false,
     "start_time": "2023-10-05T14:37:35.964752",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "读取数据集\n",
    "处理原始数据\n",
    "检查信息和说明\n",
    "检查唯一值和null\n",
    "初始特征\n",
    "根据CV结果判定是否需要加入原始数据\n",
    "\"\"\"\n",
    "class Preprocessor():\n",
    "    def __init__(self):\n",
    "        self.train    = pd.read_csv(CFG.path + f\"train.csv\", index_col = 'id');\n",
    "        self.test     = pd.read_csv(CFG.path + f\"test.csv\", index_col = 'id');\n",
    "        self.target   = CFG.target ;\n",
    "        self.original = pd.read_csv(CFG.orig_path);\n",
    "        self.conjoin_orig_data = CFG.conjoin_orig_data;\n",
    "        self.dtl_preproc_req   = CFG.dtl_preproc_req;\n",
    "        \n",
    "        self.sub_fl   = pd.read_csv(CFG.path + f\"sample_submission.csv\");\n",
    "        \n",
    "        PrintColor(f\"Data shapes - train-test-original = {self.train.shape} {self.test.shape} {self.original.shape}\");\n",
    "        \n",
    "        for tbl in [self.train, self.original, self.test]:\n",
    "            tbl.columns = tbl.columns.str.replace(r\"\\(|\\)\",\"\", regex = True);\n",
    "        \n",
    "        PrintColor(f\"\\nTrain set head\", color = Fore.GREEN);\n",
    "        display(self.train.head(5).style.format(precision = 3));\n",
    "        PrintColor(f\"\\nTest set head\", color = Fore.GREEN);\n",
    "        display(self.test.head(5).style.format(precision = 3));\n",
    "        PrintColor(f\"\\nOriginal set head\", color = Fore.GREEN);\n",
    "        display(self.original.head(5).style.format(precision = 3));\n",
    "                 \n",
    "        # 重置原始数据索引\n",
    "        self.original.index = range(len(self.original));\n",
    "        self.original.index+= max(self.test.index) + 1;\n",
    "        self.original.index.name = 'id';\n",
    "        \n",
    "        # 更改原始数据列顺序以训练数据结构\n",
    "        self.original = self.original.reindex(self.train.columns, axis=1);\n",
    "    \n",
    "    # 为train、test、original添加分数列\n",
    "    def _AddSourceCol(self):\n",
    "        self.train['Source'] = \"Competition\";\n",
    "        self.test['Source']  = \"Competition\";\n",
    "        self.original['Source'] = 'Original';\n",
    "        \n",
    "        self.strt_ftre = self.test.columns;\n",
    "        return self;\n",
    "    \n",
    "    # 将数值列的字符串替换为空值\n",
    "    def _CleanOrigNulls(self):\n",
    "        for col in ['uniq_Op', 'uniq_Opnd', 'total_Op', 'total_Opnd', 'branchCount']:\n",
    "            self.original.loc[self.original[col] == r\"?\",col] = np.NaN;\n",
    "            self.original[col] = self.original[col].astype(np.float64);\n",
    "        return self;\n",
    "    \n",
    "    # 数据基本信息展示\n",
    "    def _CollateInfoDesc(self):\n",
    "        if self.dtl_preproc_req == \"Y\":\n",
    "            PrintColor(f\"\\n{'-'*20} Information and description {'-'*20}\\n\", color = Fore.MAGENTA);\n",
    "\n",
    "            # Creating dataset information and description:\n",
    "            for lbl, df in {'Train': self.train, 'Test': self.test, 'Original': self.original}.items():\n",
    "                PrintColor(f\"\\n{lbl} description\\n\");\n",
    "                display(df.describe(percentiles= [0.05, 0.25, 0.50, 0.75, 0.9, 0.95, 0.99]).\\\n",
    "                        transpose().\\\n",
    "                        drop(columns = ['count'], errors = 'ignore').\\\n",
    "                        drop([CFG.target], axis=0, errors = 'ignore').\\\n",
    "                        style.format(formatter = '{:,.2f}').\\\n",
    "                        background_gradient(cmap = 'Blues')\n",
    "                       );\n",
    "\n",
    "                PrintColor(f\"\\n{lbl} information\\n\");\n",
    "                display(df.info());\n",
    "                collect();\n",
    "        return self;\n",
    "    \n",
    "    # 展示各列唯一值个数\n",
    "    def _CollateUnqNull(self):\n",
    "        if self.dtl_preproc_req == \"Y\":\n",
    "            PrintColor(f\"\\nUnique and null values\\n\");\n",
    "            _ = pd.concat([self.train[self.strt_ftre].nunique(), \n",
    "                           self.test[self.strt_ftre].nunique(), \n",
    "                           self.original[self.strt_ftre].nunique(),\n",
    "                           self.train[self.strt_ftre].isna().sum(axis=0),\n",
    "                           self.test[self.strt_ftre].isna().sum(axis=0),\n",
    "                           self.original[self.strt_ftre].isna().sum(axis=0)\n",
    "                          ], \n",
    "                          axis=1);\n",
    "            _.columns = ['Train_Nunq', 'Test_Nunq', 'Original_Nunq', \n",
    "                         'Train_Nulls', 'Test_Nulls', 'Original_Nulls'\n",
    "                        ];\n",
    "\n",
    "            display(_.T.style.background_gradient(cmap = 'Blues', axis=1).\\\n",
    "                    format(formatter = '{:,.0f}')\n",
    "                   );\n",
    "            \n",
    "        return self;\n",
    "    \n",
    "    # 开始对数据进行处理\n",
    "    def DoPreprocessing(self):\n",
    "        self._CleanOrigNulls();\n",
    "        self._AddSourceCol();\n",
    "        self._CollateInfoDesc();\n",
    "        self._CollateUnqNull();\n",
    "        \n",
    "        return self; \n",
    "    \n",
    "    # 加入原始数据\n",
    "    def ConjoinTrainOrig(self):\n",
    "        if self.conjoin_orig_data == \"Y\":\n",
    "            PrintColor(f\"Train shape before conjoining with original = {self.train.shape}\");\n",
    "            train = pd.concat([self.train, self.original], axis=0, ignore_index = True);\n",
    "            PrintColor(f\"Train shape after conjoining with original= {train.shape}\");\n",
    "            \n",
    "            train = train.drop_duplicates();\n",
    "            PrintColor(f\"Train shape after de-duping = {train.shape}\");\n",
    "            \n",
    "            train.index = range(len(train));\n",
    "            train.index.name = 'id';\n",
    "        \n",
    "        else:\n",
    "            PrintColor(f\"We are using the competition training data only\");\n",
    "            train = self.train;\n",
    "        return train;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 2.520585,
     "end_time": "2023-10-05T14:37:38.732754",
     "exception": false,
     "start_time": "2023-10-05T14:37:36.212169",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "pp = Preprocessor();\n",
    "pp.DoPreprocessing();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.232039,
     "end_time": "2023-10-05T14:37:39.075914",
     "exception": false,
     "start_time": "2023-10-05T14:37:38.843875",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 在2个指定数据集之间执行对抗性CV\n",
    "def Do_AdvCV(df1:pd.DataFrame, df2:pd.DataFrame, source1:str, source2:str):\n",
    "    # 对每一列进行检验\n",
    "    ftre = pp.test.select_dtypes(include = np.number).\\\n",
    "    drop(columns = ['id', \"Source\"], errors = 'ignore').columns;\n",
    "    adv_cv = {};\n",
    "    \n",
    "    for col in ftre:\n",
    "        # 随机数种子\n",
    "        shuffle_state = np.random.randint(low = 10, high = 100, size= 1);\n",
    "\n",
    "        full_df = \\\n",
    "        pd.concat([df1[[col]].assign(Source = source1), df2[[col]].assign(Source = source2)], \n",
    "                  axis=0, ignore_index = True).\\\n",
    "        sample(frac = 1.00, random_state = shuffle_state);\n",
    "\n",
    "        full_df = full_df.assign(Source_Nb = full_df['Source'].eq(source2).astype(np.int8));\n",
    "\n",
    "        # Checking for adversarial CV:-\n",
    "        model = LGBMClassifier(random_state = CFG.state, max_depth = 6, learning_rate = 0.05);\n",
    "        cv    = all_cv['SKF'];\n",
    "        score = np.mean(cross_val_score(model, \n",
    "                                        full_df[[col]], \n",
    "                                        full_df.Source_Nb, \n",
    "                                        scoring= 'roc_auc', \n",
    "                                        cv     = cv)\n",
    "                       );\n",
    "        # 添加得分情况\n",
    "        adv_cv.update({col: round(score, 4)});\n",
    "        collect();\n",
    "    \n",
    "    del ftre;\n",
    "    \n",
    "    fig, ax = plt.subplots(1,1,figsize = (12, 5));\n",
    "    pd.Series(adv_cv).plot.bar(color = 'tab:blue', ax = ax);\n",
    "    ax.axhline(y = 0.60, color = 'red', linewidth = 2.75);\n",
    "    ax.grid(**CFG.grid_specs); \n",
    "    plt.yticks(np.arange(0.0, 0.81, 0.05));\n",
    "    plt.show();\n",
    "    \n",
    "# 进行对抗性CV\n",
    "if CFG.adv_cv_req == \"Y\":\n",
    "    PrintColor(f\"\\n---------- Adversarial CV - Train vs Original ----------\\n\", \n",
    "               color = Fore.MAGENTA);\n",
    "    Do_AdvCV(df1 = pp.train, df2 = pp.original, source1 = 'Train', source2 = 'Original');\n",
    "    \n",
    "    PrintColor(f\"\\n---------- Adversarial CV - Train vs Test ----------\\n\", \n",
    "               color = Fore.MAGENTA);\n",
    "    Do_AdvCV(df1 = pp.train, df2 = pp.test, source1 = 'Train', source2 = 'Test');\n",
    "    \n",
    "    PrintColor(f\"\\n---------- Adversarial CV - Original vs Test ----------\\n\", \n",
    "               color = Fore.MAGENTA);\n",
    "    Do_AdvCV(df1 = pp.original, df2 = pp.test, source1 = 'Original', source2 = 'Test');   \n",
    "    \n",
    "if CFG.adv_cv_req == \"N\":\n",
    "    PrintColor(f\"\\nAdversarial CV is not needed\\n\", color = Fore.RED);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.411439,
     "end_time": "2023-10-05T14:37:39.503065",
     "exception": false,
     "start_time": "2023-10-05T14:37:39.091626",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# （训练数据+原始数据）、测试集、不包含目标列的其他列名\n",
    "train, test, strt_ftre = pp.ConjoinTrainOrig(), pp.test.copy(deep = True), deepcopy(pp.strt_ftre);\n",
    "# 类别列\n",
    "cat_cols  = test.select_dtypes(include = 'object').columns[:-1];\n",
    "# 连续数值列\n",
    "cont_cols = test.drop(columns = [], errors = 'ignore').select_dtypes(exclude = 'object').columns;\n",
    "\n",
    "# 若删除训练集中的空值\n",
    "if CFG.drop_nulls == \"Y\":\n",
    "    train = train.dropna();\n",
    "else: \n",
    "    pass;\n",
    "\n",
    "PrintColor(f\"\\nCategory columns\\n\");\n",
    "display(cat_cols);\n",
    "PrintColor(f\"\\nContinuous columns\\n\");\n",
    "display(np.array(cont_cols));\n",
    "PrintColor(f\"\\nAll columns\\n\");\n",
    "display(strt_ftre);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.628334,
     "end_time": "2023-10-05T14:37:40.274538",
     "exception": false,
     "start_time": "2023-10-05T14:37:39.646204",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 绘制训练数据和原始数据的目标分布\n",
    "if CFG.ftre_plots_req == \"Y\":\n",
    "    \n",
    "    fig, axes = plt.subplots(1,2, figsize = (12, 5), sharey = True, gridspec_kw = {'wspace': 0.35});\n",
    "\n",
    "    for i, df in tqdm(enumerate([pp.train, pp.original]), \"Target balance ---> \"):\n",
    "        ax= axes[i];\n",
    "        a = df[CFG.target].value_counts(normalize = True);\n",
    "        _ = ax.pie(x = a , labels = a.index.values, \n",
    "                   explode      = [0.0, 0.2], \n",
    "                   startangle   = 40, \n",
    "                   shadow       = True, \n",
    "                   colors       = ['#3377ff', '#66ffff'], \n",
    "                   textprops    = {'fontsize': 7, 'fontweight': 'bold', 'color': 'black'},\n",
    "                   pctdistance  = 0.60, \n",
    "                   autopct = '%1.1f%%'\n",
    "                  );\n",
    "        df_name = 'Train' if i == 0 else \"Original\";\n",
    "        _ = ax.set_title(f\"\\n{df_name} data\\n\", **CFG.title_specs);\n",
    "\n",
    "    plt.tight_layout();\n",
    "    plt.show();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.468978,
     "end_time": "2023-10-05T14:37:40.761940",
     "exception": false,
     "start_time": "2023-10-05T14:37:40.292962",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 显示分布具体数值\n",
    "if CFG.ftre_plots_req == \"Y\":\n",
    "    fig, axes = plt.subplots(1,2, figsize = (12, 4), gridspec_kw = {'wspace': 0.2});\n",
    "    \n",
    "    for i, (lbl, df) in enumerate({\"Train\": pp.train, \"Original\": pp.original}.items()):\n",
    "        ax = axes[i];\n",
    "        c = ['#3377ff', '#6699cc'];\n",
    "        df.groupby(CFG.target).size().plot.bar(ax = ax, color = c[i]);\n",
    "        ax.set_title(f\"Target interaction - {lbl} set\", **CFG.title_specs);\n",
    "        ax.set(xlabel = \"\");\n",
    "        \n",
    "    plt.tight_layout();\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 40.941624,
     "end_time": "2023-10-05T14:38:21.756325",
     "exception": false,
     "start_time": "2023-10-05T14:37:40.814701",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 对比训练数据、测试数据、原始数据各特征分布差异\n",
    "if CFG.ftre_plots_req == \"Y\":\n",
    "    df = pd.concat([pp.train[cont_cols].assign(Source = 'Train'), \n",
    "                    pp.test[cont_cols].assign(Source = 'Test'),\n",
    "                    pp.original[cont_cols].assign(Source = \"Original\")\n",
    "                   ], \n",
    "                   axis=0, ignore_index = True\n",
    "                  );\n",
    "    \n",
    "    fig, axes = plt.subplots(len(cont_cols), 4 ,figsize = (16, len(cont_cols) * 4.2), \n",
    "                             gridspec_kw = {'hspace': 0.35, 'wspace': 0.3, 'width_ratios': [0.80, 0.20, 0.20, 0.20]});\n",
    "    \n",
    "    for i,col in enumerate(cont_cols):\n",
    "        ax = axes[i,0];\n",
    "        sns.kdeplot(data = df[[col, 'Source']], x = col, hue = 'Source', \n",
    "                    palette = ['#0039e6', '#ff5500', '#00b300'], \n",
    "                    ax = ax, linewidth = 2.1\n",
    "                   );\n",
    "        ax.set_title(f\"\\n{col}\", **CFG.title_specs);\n",
    "        ax.grid(**CFG.grid_specs);\n",
    "        ax.set(xlabel = '', ylabel = '');\n",
    "        \n",
    "        ax = axes[i,1];\n",
    "        sns.boxplot(data = df.loc[df.Source == 'Train', [col]], y = col, width = 0.25,\n",
    "                    color = '#33ccff', saturation = 0.90, linewidth = 0.90, \n",
    "                    fliersize= 2.25,\n",
    "                    ax = ax);\n",
    "        ax.set(xlabel = '', ylabel = '');\n",
    "        ax.set_title(f\"Train\", **CFG.title_specs);\n",
    "        \n",
    "        ax = axes[i,2];\n",
    "        sns.boxplot(data = df.loc[df.Source == 'Test', [col]], y = col, width = 0.25, fliersize= 2.25,\n",
    "                    color = '#80ffff', saturation = 0.6, linewidth = 0.90, \n",
    "                    ax = ax); \n",
    "        ax.set(xlabel = '', ylabel = '');\n",
    "        ax.set_title(f\"Test\", **CFG.title_specs);\n",
    "        \n",
    "        ax = axes[i,3];\n",
    "        sns.boxplot(data = df.loc[df.Source == 'Original', [col]], y = col, width = 0.25, fliersize= 2.25,\n",
    "                    color = '#99ddff', saturation = 0.6, linewidth = 0.90, \n",
    "                    ax = ax); \n",
    "        ax.set(xlabel = '', ylabel = '');\n",
    "        ax.set_title(f\"Original\", **CFG.title_specs);\n",
    "              \n",
    "    plt.suptitle(f\"\\nDistribution analysis- continuous columns\\n\", **CFG.title_specs, \n",
    "                 y = 0.905, x = 0.50\n",
    "                );\n",
    "    plt.tight_layout();\n",
    "    plt.show();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.498806,
     "end_time": "2023-10-05T14:38:22.287618",
     "exception": false,
     "start_time": "2023-10-05T14:38:21.788812",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 特征间的偏度\n",
    "skew_df = pd.DataFrame(index = strt_ftre[0:-1]);\n",
    "for col, df in {\"Train\": pp.train, \"Test\": pp.test, \"Original\": pp.original}.items():\n",
    "    skew_df = \\\n",
    "    pd.concat([skew_df, \n",
    "               df.drop(columns = [CFG.target, \"Source\", \"id\"], errors = \"ignore\").skew()],\n",
    "              axis=1).rename({0: col}, axis=1);\n",
    "\n",
    "PrintColor(f\"\\nSkewness across independent features\\n\");\n",
    "display(skew_df.transpose().style.format(precision = 2).background_gradient(\"Pastel2\"));\n",
    "\n",
    "del skew_df;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 10.080588,
     "end_time": "2023-10-05T14:38:32.468708",
     "exception": false,
     "start_time": "2023-10-05T14:38:22.388120",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 绘制训练数据、测试数据、原始数据，pearson和spearman相关系数热力图\n",
    "def MakeCorrPlot(df: pd.DataFrame, data_label:str, figsize = (30, 9)):\n",
    "    \"\"\"\n",
    "    This function develops the correlation plots for the given dataset\n",
    "    \"\"\";\n",
    "    \n",
    "    fig, axes = plt.subplots(1,2, figsize = figsize, gridspec_kw = {'hspace': 0.2, 'wspace': 0.1},\n",
    "                             sharey = True\n",
    "                            );\n",
    "    \n",
    "    for i, method in enumerate(['pearson', 'spearman']):\n",
    "        corr_ = df.drop(columns = ['id', 'Source'], errors = 'ignore').corr(method = method);\n",
    "        ax = axes[i];\n",
    "        sns.heatmap(data = corr_,  \n",
    "                    annot= True,\n",
    "                    fmt= '.2f', \n",
    "                    cmap = 'Blues',\n",
    "                    annot_kws= {'fontweight': 'bold','fontsize': 6.75}, \n",
    "                    linewidths= 1.5, \n",
    "                    linecolor='white', \n",
    "                    cbar= False, \n",
    "                    mask= np.triu(np.ones_like(corr_)),\n",
    "                    ax= ax\n",
    "                   );\n",
    "        ax.set_title(f\"\\n{method.capitalize()} correlation- {data_label}\\n\", **CFG.title_specs);\n",
    "        \n",
    "    collect();\n",
    "    print();\n",
    "\n",
    "# Implementing correlation analysis:-\n",
    "if CFG.ftre_imp_req == \"Y\":\n",
    "    for lbl, df in {\"Train\": pp.train, \n",
    "                    \"Test\": pp.test, \n",
    "                    \"Original\": pp.original\n",
    "                   }.items():\n",
    "        MakeCorrPlot(df = df, data_label = lbl, figsize = (22,6.5));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.556974,
     "end_time": "2023-10-05T14:38:33.344782",
     "exception": false,
     "start_time": "2023-10-05T14:38:32.787808",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 此类使用简单交互为现有数据添加特征\n",
    "class Xformer(TransformerMixin, BaseEstimator):\n",
    "    def __init__(self): \n",
    "        pass\n",
    "    \n",
    "    def fit(self, X, y= None, **params):\n",
    "        return self;\n",
    "    \n",
    "    # 将特征添加到现有数据中\n",
    "    def transform(self, X, y= None, **params):\n",
    "        df    = X.copy(); \n",
    "        \n",
    "        df['mean_bnv']         = (df['n'] + df['v'] + df['b']) /3;\n",
    "        df['mean_uniqOpOpend'] = (df['uniq_Op'] + df['uniq_Opnd']) /2;\n",
    "        df['mean_totOpOpend']  = (df['total_Op'] + df['total_Opnd']) /2;\n",
    "        df['mean_brcntvg']     = (df['branchCount'] + df['vg']) / 2;\n",
    "        \n",
    "        self.op_cols = df.columns;  \n",
    "        return df;\n",
    "    \n",
    "    def get_feature_names_in(self, X, y=None, **params): \n",
    "        return self.ip_cols;    \n",
    "    \n",
    "    def get_feature_names_out(self, X, y=None, **params): \n",
    "        return self.op_cols;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.063188,
     "end_time": "2023-10-05T14:38:33.452571",
     "exception": false,
     "start_time": "2023-10-05T14:38:33.389383",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 数据缩放类\n",
    "class Scaler(TransformerMixin, BaseEstimator):\n",
    "    def __init__(self, scl_method: str, scale_req: str, scl_cols):\n",
    "        self.scl_method = scl_method;\n",
    "        self.scale_req  = scale_req;\n",
    "        self.scl_cols   = scl_cols;\n",
    "    \n",
    "    # 计算训练集缩放参数\n",
    "    def fit(self,X, y=None, **params):        \n",
    "        self.params          = X[self.scl_cols].describe(percentiles = [0.25, 0.50, 0.75]).drop(['count'], axis=0).T;\n",
    "        self.params['iqr']   = self.params['75%'] - self.params['25%'];\n",
    "        self.params['range'] = self.params['max'] - self.params['min'];\n",
    "        \n",
    "        return self;\n",
    "    \n",
    "    # 使用特定方法缩放列\n",
    "    def transform(self,X, y=None, **params):  \n",
    "        df = X.copy();\n",
    "        if self.scale_req == \"Y\":\n",
    "            if CFG.scl_method == \"Z\":\n",
    "                df[self.scl_cols] = (df[self.scl_cols].values - self.params['mean'].values) / self.params['std'].values;\n",
    "            elif CFG.scl_method == \"Robust\":\n",
    "                df[self.scl_cols] = (df[self.scl_cols].values - self.params['50%'].values) / self.params['iqr'].values;\n",
    "            elif CFG.scl_method == \"MinMax\":\n",
    "                df[self.scl_cols] = (df[self.scl_cols].values - self.params['min'].values) / self.params['range'].values;\n",
    "        else:\n",
    "            PrintColor(f\"Scaling is not needed\", color = Fore.RED);\n",
    "    \n",
    "        return df;\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.493985,
     "end_time": "2023-10-05T14:38:33.992679",
     "exception": false,
     "start_time": "2023-10-05T14:38:33.498694",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "PrintColor(f\"\\n{'='* 20} Data transformation {'='* 20} \\n\");\n",
    "\n",
    "ytrain = train[CFG.target].astype(np.int8);\n",
    "Xtrain = train.drop(columns = [CFG.target]);\n",
    "Xtest  = test.copy(deep = True);\n",
    "\n",
    "PrintColor(f\"\\n---> Train data\\n\");\n",
    "display(Xtrain.head(5).style.format(precision = 2));\n",
    "PrintColor(f\"\\n---> Test data\\n\");\n",
    "display(Xtest.head(5).style.format(precision = 2));\n",
    "\n",
    "xform = Pipeline(steps = [(\"Xform\", Xformer())]);\n",
    "Xtrain = xform.fit_transform(Xtrain, ytrain);\n",
    "Xtest  = xform.transform(Xtest);\n",
    "\n",
    "PrintColor(f\"\\n---> Train data columns after data pipeline\\n\");\n",
    "pprint(Xtrain.columns);\n",
    "\n",
    "PrintColor(f\"\\n---> Test data columns after data pipeline\\n\");\n",
    "pprint(Xtest.columns);\n",
    "\n",
    "PrintColor(f\"\\n---> Train-test shape after pipeline = {Xtrain.shape} {Xtest.shape}\");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.423486,
     "end_time": "2023-10-05T14:38:34.554400",
     "exception": false,
     "start_time": "2023-10-05T14:38:34.130914",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 各模型参数\n",
    "Mdl_Master = \\\n",
    "{'CBC': CatBoostClassifier(**{'task_type'           : \"GPU\" if CFG.gpu_switch == \"ON\" else \"CPU\",\n",
    "                              'objective'           : 'Logloss',\n",
    "                              'loss_function'       : 'Logloss',\n",
    "                              'eval_metric'         : 'AUC',\n",
    "                              'bagging_temperature' : 0.20,\n",
    "                              'colsample_bylevel'   : 0.75,\n",
    "                              'iterations'          : 1000,\n",
    "                              'learning_rate'       : 0.08,\n",
    "                              'od_wait'             : 14,\n",
    "                              'max_depth'           : 6,\n",
    "                              'l2_leaf_reg'         : 0.85,\n",
    "                              'min_data_in_leaf'    : 15,\n",
    "                              'random_strength'     : 0.70, \n",
    "                              'max_bin'             : 120,\n",
    "                              'verbose'             : 0,\n",
    "                              'use_best_model'      : True,\n",
    "                           }\n",
    "                         ), \n",
    "\n",
    "  'LGBM1C': LGBMClassifier(**{'device'           : \"gpu\" if CFG.gpu_switch == \"ON\" else \"cpu\",\n",
    "                             'objective'         : 'binary',\n",
    "                             'metric'            : 'auc',\n",
    "                             'boosting_type'     : 'gbdt',\n",
    "                             'random_state'      : CFG.state,\n",
    "                             'colsample_bytree'  : 0.5,\n",
    "                             'subsample'         : 0.65,\n",
    "                             'learning_rate'     : 0.065,\n",
    "                             'max_depth'         : 6,\n",
    "                             'n_estimators'      : 1000,\n",
    "                             'num_leaves'        : 110,                    \n",
    "                             'reg_alpha'         : 0.01,\n",
    "                             'reg_lambda'        : 1.75,\n",
    "                             'verbose'           : -1,\n",
    "                          }\n",
    "                       ),\n",
    " \n",
    "   'LGBM2C': LGBMClassifier(**{'device'           : \"gpu\" if CFG.gpu_switch == \"ON\" else \"cpu\",\n",
    "                              'objective'         : 'binary',\n",
    "                              'metric'            : 'auc',\n",
    "                              'boosting_type'     : 'goss',\n",
    "                              'random_state'      : CFG.state,\n",
    "                              'colsample_bytree'  : 0.50,\n",
    "                              'subsample'         : 0.70,\n",
    "                              'learning_rate'     : 0.0625,\n",
    "                              'max_depth'         : 6,\n",
    "                              'n_estimators'      : 1000,\n",
    "                              'num_leaves'        : 110,                    \n",
    "                              'reg_alpha'         : 0.0001,\n",
    "                              'reg_lambda'        : 2.0,\n",
    "                              'verbose'           : -1,\n",
    "                          }\n",
    "                       ),\n",
    " \n",
    "\n",
    "  'XGBC': XGBClassifier(**{'tree_method'        : \"gpu_hist\" if CFG.gpu_switch == \"ON\" else \"hist\",\n",
    "                           'objective'          : 'binary:logistic',\n",
    "                           'random_state'       : CFG.state,\n",
    "                           'colsample_bytree'   : 0.7,\n",
    "                           'learning_rate'      : 0.0655,\n",
    "                           'max_depth'          : 6,\n",
    "                           'n_estimators'       : 1100,                         \n",
    "                           'reg_alpha'          : 0.0001,\n",
    "                           'reg_lambda'         : 2.25,\n",
    "                           'min_child_weight'   : 18,\n",
    "                           'early_stopping_rounds' : CFG.nbrnd_erly_stp,\n",
    "                           'eval_metric'        : \"auc\",\n",
    "                        }\n",
    "                       ),\n",
    " \n",
    "  \"HGBC\" : HGBC(loss              = 'log_loss',\n",
    "                learning_rate     = 0.07,\n",
    "                early_stopping    = True,\n",
    "                max_iter          = 200,\n",
    "                max_depth         = 6,\n",
    "                min_samples_leaf  = 20,\n",
    "                l2_regularization = 1.75,\n",
    "                scoring           = myscorer,\n",
    "                random_state      = CFG.state,\n",
    "               ),\n",
    " \n",
    "  \"RFC\" : RFC(n_estimators     = 150,\n",
    "              criterion        = 'gini',\n",
    "              max_depth        = 7,\n",
    "              min_samples_leaf = 12,\n",
    "              max_features     = 'sqrt',\n",
    "              bootstrap        = True,\n",
    "              oob_score        = True,\n",
    "              n_jobs           = -1, \n",
    "              random_state     = CFG.state,\n",
    "              verbose          = 0,\n",
    "             ),\n",
    " \n",
    "  \"GBC\" : GBC(loss           = 'log_loss',\n",
    "              learning_rate  = 0.0525,\n",
    "              n_estimators   = 250,\n",
    "              subsample      = 0.45,\n",
    "              min_samples_leaf = 18,\n",
    "              max_depth      = 6,\n",
    "              random_state   = CFG.state,\n",
    "              n_iter_no_change = CFG.nbrnd_erly_stp,\n",
    "             ),\n",
    " \n",
    "  \"LC\"  : LC(penalty         = 'l2', \n",
    "                tol          = 0.0001, \n",
    "                C            = 0.01, \n",
    "                random_state = CFG.state,\n",
    "                max_iter     = 1000,\n",
    "                verbose      = 0,\n",
    "               ),\n",
    "\n",
    "};"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.432737,
     "end_time": "2023-10-05T14:38:35.033123",
     "exception": false,
     "start_time": "2023-10-05T14:38:34.600386",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "if CFG.ML == \"Y\":\n",
    "\n",
    "    # 初始化模型输出\n",
    "    if CFG.drop_nulls == \"Y\":\n",
    "        methods      = list(Mdl_Master.keys());\n",
    "    else:\n",
    "        methods = ['XGBC', \"LGBM1C\", \"LGBM2C\", \"CBC\", \"HGBC\"];\n",
    "    \n",
    "    sel_cols  = \\\n",
    "    ['Source',\n",
    "     'loc', 'vg', 'evg', 'ivg', 'n', 'v', 'l', 'd', 'i', 'e', 'b', 't',\n",
    "     'lOCode', 'lOComment', 'lOBlank', 'locCodeAndComment', 'uniq_Op',\n",
    "     'uniq_Opnd', 'total_Op', 'total_Opnd', 'branchCount', \n",
    "     'mean_bnv', 'mean_uniqOpOpend', 'mean_totOpOpend', 'mean_brcntvg'\n",
    "    ];\n",
    "    \n",
    "    # （训练集结果）列标签：模型名称\n",
    "    OOF_Preds = pd.DataFrame(columns = methods);\n",
    "    # （测试集结果）索引：与sub保持一致；列标签：模型名称\n",
    "    Mdl_Preds = pd.DataFrame(index = pp.sub_fl['id'], columns = methods);\n",
    "    # （模型特征重要性）索引：除Source外的其他所有列\n",
    "    FtreImp   = pd.DataFrame(index = sel_cols[1:], columns = methods);\n",
    "    # （交叉检验每折分数）列标签：模型名称\n",
    "    Scores    = pd.DataFrame(columns = methods);\n",
    "\n",
    "    PrintColor(f\"\\n---> Selected model options- \");\n",
    "    pprint(methods, depth = 1, width = 100, indent = 5);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.444171,
     "end_time": "2023-10-05T14:38:35.526009",
     "exception": false,
     "start_time": "2023-10-05T14:38:35.081838",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def TrainMdl(method:str):\n",
    "    \n",
    "    # 设定全局变量\n",
    "    global Mdl_Master, Mdl_Preds, OOF_Preds, all_cv, FtreImp, Xtrain, ytrain, sel_cols; \n",
    "    \n",
    "    # 若methed包含逻辑回归、支持向量机、K邻近、MLP模型\n",
    "    if method in [\"LC\", \"SVC\", \"KNNC\", \"MLPC\"]:\n",
    "        # 进行log变换、标准化变换、取模型\n",
    "        model = Pipeline(steps = [('Log', FT(np.log1p)), \n",
    "                                  (\"S\", all_scalers[CFG.scl_method]), \n",
    "                                  (\"M\", Mdl_Master.get(method))\n",
    "                                 ]);\n",
    "    else:\n",
    "        # 直接取模型\n",
    "        model = Pipeline(steps = [(\"M\", Mdl_Master.get(method))]); \n",
    "    # 取交叉检验方式\n",
    "    cv        = all_cv.get(CFG.mdlcv_mthd);\n",
    "    # 取测试集对应列\n",
    "    Xt        = Xtest[sel_cols];\n",
    "    \n",
    "    if CFG.use_orig_allfolds == \"Y\":\n",
    "        # 从Xtrain中选择特征并筛选出Source为'Competition'的行\n",
    "        X    = Xtrain[sel_cols].query(\"Source == 'Competition'\");\n",
    "        # 获取与X的索引匹配的ytrain的子集\n",
    "        y    = ytrain.loc[ytrain.index.isin(X.index)]; \n",
    "        # 创建数据框，其中包含Source为'Original'的行和特征\n",
    "        Orig = pd.concat([Xtrain, ytrain], axis=1).query(\"Source == 'Original'\")[sel_cols];\n",
    "        \n",
    "    elif CFG.use_orig_allfolds != \"Y\":\n",
    "        # 直接使用Xtrain的特定列\n",
    "        X,y = Xtrain[sel_cols], ytrain.copy(deep = True);\n",
    "                \n",
    "    # 初始化参数        \n",
    "    test_preds = 0;\n",
    "    oof_preds  = pd.DataFrame(); \n",
    "    scores     = [];\n",
    "    ftreimp    = 0;\n",
    "    cols_drop  = ['Source'];\n",
    "    \n",
    "    # 开始执行交叉检验\n",
    "    for fold_nb, (train_idx, dev_idx) in enumerate(cv.split(X, y)):\n",
    "        # 从训练集中删除目标（Source）列\n",
    "        Xtr  = X.iloc[train_idx].drop(columns = cols_drop, errors = 'ignore');   \n",
    "        # 从验证集中筛选出'Source'为'Competition'的行\n",
    "        Xdev = X.iloc[dev_idx].loc[X.Source == \"Competition\"].\\\n",
    "        drop(columns = cols_drop, errors = 'ignore');\n",
    "        # 取训练集对应目标列\n",
    "        ytr  = y.loc[y.index.isin(Xtr.index)];\n",
    "        # 取验证集对应目标列\n",
    "        ydev = y.loc[y.index.isin(Xdev.index)];\n",
    "\n",
    "        if CFG.use_orig_allfolds == \"Y\":\n",
    "            # 将Xtr和ytr与Orig合并\n",
    "            Xtr = pd.concat([Xtr, Orig.drop(columns = [CFG.target, 'Source'], errors = 'ignore')], \n",
    "                            axis = 0, ignore_index = True);\n",
    "            ytr = pd.concat([ytr, Orig[CFG.target]], axis = 0, ignore_index = True);\n",
    "            \n",
    "        # 训练模型\n",
    "        # ‘CBC’或'CBR'模型\n",
    "        if method in ['CBC', 'CBR']:    \n",
    "            model.fit(Xtr, ytr, \n",
    "                      M__eval_set = [(Xdev, ydev)], \n",
    "                      M__verbose = 0,\n",
    "                      M__early_stopping_rounds = CFG.nbrnd_erly_stp,\n",
    "                                           ); \n",
    "        # ‘LGBM’模型\n",
    "        elif \"LGBM\" in method:\n",
    "            model.fit(Xtr, ytr, \n",
    "                      M__eval_set = [(Xdev, ydev)], \n",
    "                      M__verbose = 0,\n",
    "                      M__callbacks = [log_evaluation(0), \n",
    "                                      early_stopping(stopping_rounds = CFG.nbrnd_erly_stp, \n",
    "                                                     verbose = False,)\n",
    "                                     ]\n",
    "                     ); \n",
    "        # XGB模型\n",
    "        elif \"XGB\" in method:\n",
    "             model.fit(Xtr, ytr, M__eval_set = [(Xdev, ydev)], M__verbose = 0,);            \n",
    "        \n",
    "        # sklearn封装模型   \n",
    "        else: \n",
    "            model.fit(Xtr, ytr); \n",
    "            \n",
    "        # 收集预测结果和分数，并根据模型方法对Out-of-Fold（OOF）数据进行后处理\n",
    "        if method.upper().endswith('R'):\n",
    "            # 对于回归模型\n",
    "            dev_preds = PostProcessPred(model.predict(Xdev), \n",
    "                                        post_process= CFG.pstprcs_train);\n",
    "            test_preds = test_preds + \\\n",
    "            PostProcessPred(model.predict(Xt.drop(columns = cols_drop, errors = 'ignore')),\n",
    "                            post_process= CFG.pstprcs_train); \n",
    "        else:\n",
    "            # 对于分类模型，获取类别为1的概率\n",
    "            dev_preds = model.predict_proba(Xdev)[:,1];\n",
    "            test_preds = test_preds + \\\n",
    "            model.predict_proba(Xt.drop(columns = cols_drop, errors = 'ignore'))[:,1];            \n",
    "        # AUC指标\n",
    "        score = ScoreMetric(ydev.values.flatten(), dev_preds);\n",
    "        scores.append(score); \n",
    "         \n",
    "        # 将得分保存到Scores数据框中\n",
    "        Scores.loc[fold_nb, method] = np.round(score, decimals= 6);\n",
    "        # 将OOF预测值添加到oof_preds中\n",
    "        oof_preds = pd.concat([oof_preds,\n",
    "                               pd.DataFrame(index   = Xdev.index, \n",
    "                                            data    = dev_preds,\n",
    "                                            columns = [method])\n",
    "                              ],axis=0, ignore_index= False\n",
    "                             );  \n",
    "    \n",
    "        oof_preds = pd.DataFrame(oof_preds.groupby(level = 0)[method].mean());\n",
    "        oof_preds.columns = [method];\n",
    "        \n",
    "        try: \n",
    "            ftreimp += model[\"M\"].feature_importances_;\n",
    "        except: \n",
    "            ftreimp = 0;\n",
    "    # 打印模型的平均交叉验证分数\n",
    "    num_space = 10 - len(method);\n",
    "    PrintColor(f\"--> {method}{'-' * num_space} CV = {np.mean(scores):.6f}\");\n",
    "    del num_space;\n",
    "    \n",
    "    # 对OOF预测值进行后处理并保存到OOF_Preds中\n",
    "    OOF_Preds[f'{method}'] = PostProcessPred(oof_preds.values.flatten(), CFG.pstprcs_train);\n",
    "    \n",
    "    # 如果交叉验证方法是'KF'或'SKF'\n",
    "    if CFG.mdlcv_mthd in ['KF', 'SKF']:\n",
    "        Mdl_Preds[f'{method}'] = test_preds.flatten()/ CFG.n_splits; \n",
    "        FtreImp[method]        = ftreimp / CFG.n_splits;\n",
    "    else:\n",
    "        Mdl_Preds[f'{method}'] = test_preds.flatten()/ (CFG.n_splits * CFG.n_repeats); \n",
    "        FtreImp[method]        = ftreimp / (CFG.n_splits * CFG.n_repeats);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 1248.020484,
     "end_time": "2023-10-05T14:59:23.591937",
     "exception": false,
     "start_time": "2023-10-05T14:38:35.571453",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "if CFG.ML == \"Y\":\n",
    "    if CFG.ML == \"Y\": \n",
    "        PrintColor(f\"\\n{'-' * 25} ML model training {'-' * 25}\\n\", color = Fore.MAGENTA);\n",
    "        for method in tqdm(methods, \"ML models----\"): \n",
    "            TrainMdl(method);\n",
    "        clear_output();\n",
    "\n",
    "        PrintColor(f\"\\n{'-' * 20} OOF CV scores across methods {'-' * 20}\\n\", \n",
    "                   color = Fore.MAGENTA);\n",
    "        display(pd.concat([Scores.mean(axis = 0), Scores.std(axis = 0)], axis=1).\\\n",
    "                rename(columns = {0: 'Mean', 1: 'Std'}).T.\\\n",
    "                style.format(precision = 6).\\\n",
    "                background_gradient(subset = methods, cmap = 'cubehelix', axis=1)\n",
    "               );    \n",
    "    else:\n",
    "        PrintColor(f\"\\nML models are not needed\\n\", color = Fore.RED);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 7.109744,
     "end_time": "2023-10-05T14:59:31.031136",
     "exception": false,
     "start_time": "2023-10-05T14:59:23.921392",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 分析模型结果、特征重要性和校准曲线\n",
    "if CFG.ML == \"Y\":\n",
    "    fig, axes = plt.subplots(len(methods), 2, figsize = (25, len(methods) * 6),\n",
    "                             gridspec_kw = {'hspace': 0.2, 'wspace': 0.2}, \n",
    "                             width_ratios= [0.7, 0.3],\n",
    "                            );\n",
    "    \n",
    "    for i, col in enumerate(methods):\n",
    "        try: \n",
    "            ax = axes[i,0];\n",
    "        except: \n",
    "            ax = axes[0];\n",
    "        FtreImp[col].plot.barh(ax = ax, color = '#0073e6');\n",
    "        ax.set_title(f\"{col} Importances\", **CFG.title_specs);\n",
    "        ax.set(xlabel = '', ylabel = '');\n",
    "\n",
    "        try:\n",
    "            ax = axes[i,1];\n",
    "        except:\n",
    "            ax = axes[1];\n",
    "            \n",
    "        Clb.from_predictions(ytrain[0:len(OOF_Preds)], OOF_Preds[col], n_bins= 20, ref_line = True,\n",
    "                             **{'color': '#0073e6', 'linewidth': 1.2, \n",
    "                                'markersize': 3.75, 'marker': 'o', 'markerfacecolor': '#cc7a00'},\n",
    "                             ax = ax);\n",
    "        ax.set_title(f\"{col} Calibration\", **CFG.title_specs);\n",
    "        ax.set(xlabel = '', ylabel = '',);\n",
    "        ax.set_yticks(np.arange(0,1.01, 0.05), labels = np.round(np.arange(0,1.01, 0.05), 2), fontsize = 7.0);\n",
    "        ax.set_xticks(np.arange(0,1.01, 0.05), \n",
    "                      labels = np.round(np.arange(0,1.01, 0.05), 2), \n",
    "                      fontsize = 7.0, \n",
    "                      rotation = 90\n",
    "                     );\n",
    "        ax.legend('');\n",
    "            \n",
    "\n",
    "\n",
    "    plt.tight_layout();\n",
    "    plt.show();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.328884,
     "end_time": "2023-10-05T14:59:31.523474",
     "exception": false,
     "start_time": "2023-10-05T14:59:31.194590",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 集合模型\n",
    "if CFG.ensemble_req == \"Y\":\n",
    "    # 使用optuna搜寻模型最优权重\n",
    "    def Objective(trial):\n",
    "        global OOF_Preds, all_cv, ytrain, methods;\n",
    "\n",
    "        # 定义权重搜索空间\n",
    "        weights  = [trial.suggest_float(f\"M{n}\", 0.0001, 0.9999, step = 0.001) \\\n",
    "                    for n in range(len(OOF_Preds[methods].columns))\n",
    "                   ];\n",
    "\n",
    "        # 计算集合模型分数\n",
    "        scores = [];  \n",
    "        cv     = all_cv[CFG.enscv_mthd];\n",
    "        # 取各模型预测概率\n",
    "        X,y    = OOF_Preds[methods], ytrain[0: len(OOF_Preds)];\n",
    "\n",
    "        # 交叉检验\n",
    "        for fold_nb, (train_idx, dev_idx) in enumerate(cv.split(X,y)):\n",
    "            Xtr, Xdev = X.iloc[train_idx], X.iloc[dev_idx];\n",
    "            ytr, ydev = y.loc[Xtr.index],  y.loc[Xdev.index];\n",
    "            scores.append(ScoreMetric(ydev, np.average(Xdev, axis=1, weights = weights)));\n",
    "        return np.mean(scores);\n",
    "clear_output();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 467.600526,
     "end_time": "2023-10-05T15:07:19.177837",
     "exception": false,
     "start_time": "2023-10-05T14:59:31.577311",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 开始堆叠模型\n",
    "if CFG.ensemble_req == \"Y\":   \n",
    "    PrintColor(f\"\\n{'-' * 20} Creating an Optuna Ensemble {'-' * 20}\\n\", \n",
    "               color = Fore.MAGENTA);   \n",
    "    # 最大化目标值\n",
    "    study = optuna.create_study(direction  = CFG.metric_obj, \n",
    "                                study_name = \"OptunaEnsemble\", \n",
    "                                sampler    = TPESampler(seed = CFG.state)\n",
    "                               );\n",
    "    study.optimize(Objective, \n",
    "                   n_trials          = CFG.ntrials, \n",
    "                   gc_after_trial    = True,\n",
    "                   show_progress_bar = True\n",
    "                  );\n",
    "    weights       = study.best_params;\n",
    "    clear_output();\n",
    "    \n",
    "    PrintColor(f\"\\n--> Post ensemble weights\\n\");\n",
    "    pprint(weights, indent = 5, width = 10, depth = 1);\n",
    "    PrintColor(f\"\\n--> Best ensemble CV score = {study.best_value :.6f}\\n\");\n",
    "    \n",
    "    # 使用集合模型在获得测试集结果\n",
    "    sub_fl = pp.sub_fl.copy(deep = True);\n",
    "    sub_fl[CFG.target] = np.average(Mdl_Preds[methods], weights = list(weights.values()),\n",
    "                                    axis=1);\n",
    "    \n",
    "    PrintColor(f\"\\n--> Post ensemble test-set predictions\\n\");\n",
    "    display(sub_fl.head(5).style.format(precision = 5)); \n",
    "      \n",
    "    # 创建结果直方图\n",
    "    fig, ax = plt.subplots(1,1, figsize = (8,5), gridspec_kw = {'hspace': 0.2, 'wspace': 0.2});\n",
    "    _ = pd.DataFrame(np.arange(0, 1.01, 0.03), dtype = np.float16);\n",
    "\n",
    "    sns.histplot(sub_fl[CFG.target], color = '#0039e6', bins = 'auto', ax = ax);\n",
    "    ax.set_title(f\"\\nEnsemble predictions before post-processing\\n\", **CFG.title_specs);\n",
    "    ax.set(xlabel = '', ylabel = '');\n",
    "    ax.set_xticks(_.values.flatten(), labels = np.round(_.values.flatten(),2), rotation = 90);\n",
    "   \n",
    "    plt.tight_layout();\n",
    "    plt.show();\n",
    "\n",
    "    del _;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 2.935371,
     "end_time": "2023-10-05T15:07:22.168323",
     "exception": false,
     "start_time": "2023-10-05T15:07:19.232952",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "if CFG.ensemble_req == \"Y\":\n",
    "    # 保存文件并准备提交\n",
    "    public_sub = pd.read_csv(f\"/kaggle/input/ps-s3-e23-eda-model-en-es/submission.csv\", index_col = \"id\");\n",
    "    sub_fl[CFG.target] = sub_fl[CFG.target].values * 0.05 + public_sub[CFG.target].values * 0.95;\n",
    "    del public_sub;\n",
    "\n",
    "    PrintColor(f\"\\n---> Submission file after blending with public notebook\\n\");\n",
    "    display(sub_fl.head(10).style.format(precision = 3));\n",
    "    sub_fl.to_csv(f\"Submission_V{CFG.version_nb}.csv\", index = None);\n",
    "    \n",
    "if CFG.ML == \"Y\":  \n",
    "    OOF_Preds.add_prefix(f\"V{CFG.version_nb}_\").to_csv(f\"OOF_Preds_V{CFG.version_nb}.csv\");\n",
    "    Mdl_Preds.add_prefix(f\"V{CFG.version_nb}_\").to_csv(f\"Mdl_Preds_V{CFG.version_nb}.csv\"); \n",
    "    if isinstance(Scores, pd.DataFrame) == True:\n",
    "        Scores.to_csv(f\"Scores_V{CFG.version_nb}.csv\");"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
